package com.dxf.bigdata.D02_demo1

import org.apache.spark.{SparkConf, SparkContext}

object S02_WordCount {

  def doSome(sc: SparkContext) = {
    //1 读取文件,获取一行一行的数据
    val lines = sc.textFile("datas/*")
    //2 空格切分
    val words = lines.flatMap(_.split(" "))

    val wordToOne = words.map(
      word => (word, 1)
    )


    val wordGroup = wordToOne.groupBy(
      t => t._1
    )
    //4 统计
    //(hello,3),(word,2)
    val wordToCount = wordGroup.map {
      case (word, list) => {
        list.reduce(
          (t1, t2) => {
            (t1._1, t1._2 + t2._2)
          },
        )
      }
    }

    val array: Array[(String, Int)] = wordToCount.collect()

    array.foreach(println)
  }

  def main(args: Array[String]): Unit = {

    //1 建立和spark的连接

    val sparkConf = new SparkConf().setMaster("local").setAppName("wordCount")
    val sc = new SparkContext(sparkConf)


    //2 处理业务
    doSome(sc)

    //3 关闭连接
    sc.stop()


  }
}
